<!DOCTYPE html>
<html lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 5.4.0">


  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/logo.svg" color="#222">

<link rel="stylesheet" href="/css/main.css">



<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@5.15.2/css/all.min.css">
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/animate.css@3.1.1/animate.min.css">

<script class="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"littlefxc.github.io","root":"/","images":"/images","scheme":"Mist","version":"8.2.2","exturl":false,"sidebar":{"position":"left","display":"post","padding":18,"offset":12},"copycode":false,"bookmark":{"enable":false,"color":"#222","save":"auto"},"fancybox":false,"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"motion":{"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"fadeInDown","post_body":"fadeInDown","coll_header":"fadeInLeft","sidebar":"fadeInUp"}},"prism":false,"i18n":{"placeholder":"搜索...","empty":"没有找到任何搜索结果：${query}","hits_time":"找到 ${hits} 个搜索结果（用时 ${time} 毫秒）","hits":"找到 ${hits} 个搜索结果"},"path":"/search.xml","localsearch":{"enable":true,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false}};
  </script>
<meta name="description" content="1 前言Flink 是一个以 Java 及 Scala 作为开发语言的开源大数据项目，代码开源在 GitHub 上，并使用 Maven 来编译和构建项目。 关于开发测试环境，Mac OS、Linux 系统或者 Windows 都可以。如果使用的是 Windows 10 系统，建议使用 Windows 10 系统的 Linux 子系统来编译和运行。">
<meta property="og:type" content="article">
<meta property="og:title" content="flink集群安装">
<meta property="og:url" content="http://littlefxc.github.io/2021/04/08/flink%E9%9B%86%E7%BE%A4%E5%AE%89%E8%A3%85/index.html">
<meta property="og:site_name" content="一年春又来">
<meta property="og:description" content="1 前言Flink 是一个以 Java 及 Scala 作为开发语言的开源大数据项目，代码开源在 GitHub 上，并使用 Maven 来编译和构建项目。 关于开发测试环境，Mac OS、Linux 系统或者 Windows 都可以。如果使用的是 Windows 10 系统，建议使用 Windows 10 系统的 Linux 子系统来编译和运行。">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/3.png">
<meta property="og:image" content="https://gitee.com/littlefxc/oss/raw/master/images/image-20210409111154176.png">
<meta property="article:published_time" content="2021-04-08T08:11:55.000Z">
<meta property="article:modified_time" content="2021-04-09T08:22:27.599Z">
<meta property="article:author" content="一年春又来">
<meta property="article:tag" content="flink">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://gitee.com/littlefxc/oss/raw/master/images/3.png">


<link rel="canonical" href="http://littlefxc.github.io/2021/04/08/flink%E9%9B%86%E7%BE%A4%E5%AE%89%E8%A3%85/">


<script class="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : false,
    isPost : true,
    lang   : 'zh-CN'
  };
</script>
<title>flink集群安装 | 一年春又来</title>
  




  <noscript>
  <style>
  body { margin-top: 2rem; }

  .use-motion .menu-item,
  .use-motion .sidebar,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header {
    visibility: visible;
  }

  .use-motion .header,
  .use-motion .site-brand-container .toggle,
  .use-motion .footer { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle,
  .use-motion .custom-logo-image {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line {
    transform: scaleX(1);
  }

  .search-pop-overlay, .sidebar-nav { display: none; }
  .sidebar-panel { display: block; }
  </style>
</noscript>

<link rel="alternate" href="/atom.xml" title="一年春又来" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage" class="use-motion">
  <div class="headband"></div>

  <main class="main">
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="切换导航栏" role="button">
        <span class="toggle-line"></span>
        <span class="toggle-line"></span>
        <span class="toggle-line"></span>
    </div>
  </div>

  <div class="site-meta">

    <a href="/" class="brand" rel="start">
      <i class="logo-line"></i>
      <h1 class="site-title">一年春又来</h1>
      <i class="logo-line"></i>
    </a>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
        <i class="fa fa-search fa-fw fa-lg"></i>
    </div>
  </div>
</div>



<nav class="site-nav">
  <ul class="main-menu menu">
        <li class="menu-item menu-item-home"><a href="/" rel="section"><i class="home                          //首页 fa-fw"></i>首页</a></li>
        <li class="menu-item menu-item-archives"><a href="/archives/" rel="section"><i class="archive          //归档 fa-fw"></i>归档</a></li>
        <li class="menu-item menu-item-categories"><a href="/categories/" rel="section"><i class="th           //分类 fa-fw"></i>分类</a></li>
        <li class="menu-item menu-item-tags"><a href="/tags/" rel="section"><i class="tags                     //标签 fa-fw"></i>标签</a></li>
      <li class="menu-item menu-item-search">
        <a role="button" class="popup-trigger"><i class="fa fa-search fa-fw"></i>搜索
        </a>
      </li>
  </ul>
</nav>



  <div class="search-pop-overlay">
    <div class="popup search-popup"><div class="search-header">
  <span class="search-icon">
    <i class="fa fa-search"></i>
  </span>
  <div class="search-input-container">
    <input autocomplete="off" autocapitalize="off" maxlength="80"
           placeholder="搜索..." spellcheck="false"
           type="search" class="search-input">
  </div>
  <span class="popup-btn-close" role="button">
    <i class="fa fa-times-circle"></i>
  </span>
</div>
<div class="search-result-container no-result">
  <div class="search-result-icon">
    <i class="fa fa-spinner fa-pulse fa-5x"></i>
  </div>
</div>

    </div>
  </div>

</div>
        
  
  <div class="toggle sidebar-toggle" role="button">
    <span class="toggle-line"></span>
    <span class="toggle-line"></span>
    <span class="toggle-line"></span>
  </div>

  <aside class="sidebar">

    <div class="sidebar-inner sidebar-nav-active sidebar-toc-active">
      <ul class="sidebar-nav">
        <li class="sidebar-nav-toc">
          文章目录
        </li>
        <li class="sidebar-nav-overview">
          站点概览
        </li>
      </ul>

      <div class="sidebar-panel-container">
        <!--noindex-->
        <div class="post-toc-wrap sidebar-panel">
            <div class="post-toc animated"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#1-%E5%89%8D%E8%A8%80"><span class="nav-text">1 前言</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#2-%E5%9F%BA%E6%9C%AC%E6%A6%82%E5%BF%B5"><span class="nav-text">2 基本概念</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#3-%E5%AE%89%E8%A3%85%E7%8E%AF%E5%A2%83%E5%87%86%E5%A4%87"><span class="nav-text">3 安装环境准备</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#4-%E6%97%A5%E5%BF%97%E7%9A%84%E6%9F%A5%E7%9C%8B%E5%92%8C%E9%85%8D%E7%BD%AE"><span class="nav-text">4 日志的查看和配置</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#5-%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6-Flink-conf-yaml-%E8%AF%A6%E8%A7%A3"><span class="nav-text">5 配置文件 Flink-conf.yaml 详解</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#5-1-%E5%9F%BA%E7%A1%80%E9%85%8D%E7%BD%AE"><span class="nav-text">5.1 基础配置</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#5-2-%E9%AB%98%E5%8F%AF%E7%94%A8"><span class="nav-text">5.2 高可用</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#5-3-%E5%AE%B9%E9%94%99%E5%92%8C%E6%A3%80%E6%9F%A5%E7%82%B9%E9%85%8D%E7%BD%AE"><span class="nav-text">5.3 容错和检查点配置</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#5-4-Restful-%E5%92%8C-web-%E5%89%8D%E7%AB%AF%E9%85%8D%E7%BD%AE"><span class="nav-text">5.4 Restful 和 web 前端配置</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#5-5-%E9%AB%98%E7%BA%A7%E9%85%8D%E7%BD%AE"><span class="nav-text">5.5 高级配置</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#5-6-Flink-%E9%9B%86%E7%BE%A4%E5%AE%89%E5%85%A8%E9%85%8D%E7%BD%AE"><span class="nav-text">5.6 Flink 集群安全配置</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#5-7-Zookeeper-%E5%AE%89%E5%85%A8%E9%85%8D%E7%BD%AE"><span class="nav-text">5.7 Zookeeper 安全配置</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#5-8-HistoryServer"><span class="nav-text">5.8 HistoryServer</span></a></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#6-masters"><span class="nav-text">6 masters</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#7-workers"><span class="nav-text">7 workers</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#8-zoo-cfg-%E5%8F%AF%E9%80%89"><span class="nav-text">8 zoo.cfg(可选)</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#%E5%8F%82%E8%80%83%E8%B5%84%E6%BA%90"><span class="nav-text">参考资源</span></a></li></ol></div>
        </div>
        <!--/noindex-->

        <div class="site-overview-wrap sidebar-panel">
          <div class="site-author site-overview-item animated" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <p class="site-author-name" itemprop="name">一年春又来</p>
  <div class="site-description" itemprop="description"></div>
</div>
<div class="site-state-wrap site-overview-item animated">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/archives/">
        
          <span class="site-state-item-count">234</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/categories/">
          
        <span class="site-state-item-count">38</span>
        <span class="site-state-item-name">分类</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/tags/">
          
        <span class="site-state-item-count">125</span>
        <span class="site-state-item-name">标签</span></a>
      </div>
  </nav>
</div>



        </div>
      </div>
    </div>
  </aside>
  <div class="sidebar-dimmer"></div>


    </header>

    
  <div class="back-to-top" role="button">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>

<noscript>
  <div class="noscript-warning">Theme NexT works best with JavaScript enabled</div>
</noscript>


    <div class="main-inner post posts-expand">


  


<div class="post-block">
  
  

  <article itemscope itemtype="http://schema.org/Article" class="post-content" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://littlefxc.github.io/2021/04/08/flink%E9%9B%86%E7%BE%A4%E5%AE%89%E8%A3%85/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="一年春又来">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="一年春又来">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          flink集群安装
        </h1>

        <div class="post-meta-container">
          <div class="post-meta">
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar"></i>
      </span>
      <span class="post-meta-item-text">发表于</span>

      <time title="创建时间：2021-04-08 16:11:55" itemprop="dateCreated datePublished" datetime="2021-04-08T16:11:55+08:00">2021-04-08</time>
    </span>
      <span class="post-meta-item">
        <span class="post-meta-item-icon">
          <i class="far fa-calendar-check"></i>
        </span>
        <span class="post-meta-item-text">更新于</span>
        <time title="修改时间：2021-04-09 16:22:27" itemprop="dateModified" datetime="2021-04-09T16:22:27+08:00">2021-04-09</time>
      </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-folder"></i>
      </span>
      <span class="post-meta-item-text">分类于</span>
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/" itemprop="url" rel="index"><span itemprop="name">大数据</span></a>
        </span>
    </span>

  
</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">
        <h1 id="1-前言"><a href="#1-前言" class="headerlink" title="1 前言"></a>1 前言</h1><p>Flink 是一个以 Java 及 Scala 作为开发语言的开源大数据项目，代码开源在 GitHub 上，并使用 Maven 来编译和构建项目。</p>
<p>关于开发测试环境，Mac OS、Linux 系统或者 Windows 都可以。如果使用的是 Windows 10 系统，建议使用 Windows 10 系统的 Linux 子系统来编译和运行。</p>
<span id="more"></span>

<p>Flink支持多种安装模式。</p>
<ul>
<li>local（本地）——单机模式，一般不使用 </li>
<li>standalone——独立模式，Flink自带集群，开发测试环境使用 </li>
<li>yarn——计算资源统一由Hadoop YARN管理，生产环境测试</li>
</ul>
<h1 id="2-基本概念"><a href="#2-基本概念" class="headerlink" title="2 基本概念"></a>2 基本概念</h1><p>运行 Flink 应用其实非常简单，但是在运行 Flink 应用之前，还是有必要了解 Flink 运行时的各个组件，因为这涉及到 Flink 应用的配置问题。</p>
<p>Flink 实际运行时包括两类进程（下图所示）：</p>
<ul>
<li>JobManager（又称为 JobMaster）：协调 Task 的分布式执行，包括调度 Task、协调创建 Checkpoint 以及当 Job failover 时协调各个 Task 从 Checkpoint 恢复等。</li>
<li>TaskManager（又称为 Worker）：执行 Dataflow 中的 Tasks，包括内存 Buffer 的分配、Data Stream 的传递等。</li>
</ul>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/3.png" alt="Flink Runtime 架构图"></p>
<p>Flink Runtime 架构图说明：</p>
<ul>
<li><p>当 Flink 集群启动后，首先会启动一个 JobManger 和一个或多个的 TaskManager。由 Client 提交任务给 JobManager，JobManager 再调度任务到各个 TaskManager 去执行，然后 TaskManager 将心跳和统计信息汇报给 JobManager。TaskManager 之间以流的形式进行数据的传输。上述三者均为独立的 JVM 进程。</p>
</li>
<li><p>JobManager</p>
<p>Master进程，负责Job的管理和资源的协调。包括任务调度，检查点管理，失败恢复等。</p>
<p>当然，对于集群HA模式，可以同时多个master进程，其中一个作为leader，其他作为standby。当leader失败时，会选出一个standby的master作为新的leader（通过zookeeper实现leader选举）。</p>
</li>
</ul>
<h1 id="3-安装环境准备"><a href="#3-安装环境准备" class="headerlink" title="3 安装环境准备"></a>3 安装环境准备</h1><ul>
<li>从<a target="_blank" rel="noopener" href="https://flink.apache.org/downloads.html">官网</a>下载压缩包</li>
<li>安装Java，并配置 JAVA_HOME 环境变量</li>
</ul>
<p>安装好Flink后，再来看下安装路径下的配置文件有哪些吧</p>
<p><img src="https://gitee.com/littlefxc/oss/raw/master/images/image-20210409111154176.png" alt="image-20210409111154176"></p>
<p>安装目录下主要有 flink-conf.yaml 配置、日志的配置文件、zk 配置、Flink SQL Client 配置。</p>
<h1 id="4-日志的查看和配置"><a href="#4-日志的查看和配置" class="headerlink" title="4 日志的查看和配置"></a>4 日志的查看和配置</h1><p>JobManager 和 TaskManager 的启动日志可以在 Flink binary 目录下的 Log 子目录中找到。Log 目录中以<code>flink-&#123;id&#125;-$&#123;hostname&#125;</code>为前缀的文件对应的是 JobManager 的输出，其中有三个文件：</p>
<ul>
<li><code>flink-$&#123;user&#125;-standalonesession-$&#123;id&#125;-$&#123;hostname&#125;.log</code>：代码中的日志输出</li>
<li><code>flink-$&#123;user&#125;-standalonesession-$&#123;id&#125;-$&#123;hostname&#125;.out</code>：进程执行时的stdout输出</li>
<li><code>flink-$&#123;user&#125;-standalonesession-$&#123;id&#125;-$&#123;hostname&#125;-gc.log</code>：JVM的GC的日志</li>
</ul>
<p>Log 目录中以<code>flink-&#123;id&#125;-$&#123;hostname&#125;</code>为前缀的文件对应的是 TaskManager 的输出，也包括三个文件，和 JobManager 的输出一致。</p>
<p>日志的配置文件在 Flink binary 目录的 conf 子目录下，其中：</p>
<ul>
<li><code>log4j-cli.properties</code>：用 Flink 命令行时用的 log 配置，比如执行“ flink run”命令</li>
<li><code>log4j-yarn-session.properties</code>：用 <code>yarn-session.sh</code> 启动时命令行执行时用的 log 配置</li>
<li><code>log4j.properties</code>：无论是 Standalone 还是 Yarn 模式，JobManager 和 TaskManager 上用的 log 配置都是 log4j.properties</li>
</ul>
<p>这三个<code>log4j.*properties</code>文件分别有三个<code>logback.*xml</code>文件与之对应，如果想使用 Logback 的同学，之需要把与之对应的<code>log4j.*properties</code>文件删掉即可，对应关系如下：</p>
<ul>
<li><code>log4j-cli.properties -&gt; logback-console.xml</code></li>
<li><code>log4j-yarn-session.properties -&gt; logback-yarn.xml</code></li>
<li><code>log4j.properties -&gt; logback.xml</code></li>
</ul>
<p>需要注意的是，“flink-{id}-和{user}-taskexecutor-{hostname}”都带有“，{id}”表示本进程在本机上该角色（JobManager 或 TaskManager）的所有进程中的启动顺序，默认从 0 开始。</p>
<h1 id="5-配置文件-Flink-conf-yaml-详解"><a href="#5-配置文件-Flink-conf-yaml-详解" class="headerlink" title="5 配置文件 Flink-conf.yaml 详解"></a>5 配置文件 Flink-conf.yaml 详解</h1><h2 id="5-1-基础配置"><a href="#5-1-基础配置" class="headerlink" title="5.1 基础配置"></a>5.1 基础配置</h2><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># jobManager 的IP地址</span></span><br><span class="line"><span class="attr">jobmanager.rpc.address:</span> <span class="string">localhost</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># JobManager 的端口号</span></span><br><span class="line"><span class="attr">jobmanager.rpc.port:</span> <span class="number">6123</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># JobManager JVM heap 内存大小</span></span><br><span class="line"><span class="attr">jobmanager.heap.size:</span> <span class="string">1024m</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># TaskManager JVM heap 内存大小</span></span><br><span class="line"><span class="attr">taskmanager.heap.size:</span> <span class="string">1024m</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每个 TaskManager 提供的任务 slots 数量大小</span></span><br><span class="line"><span class="attr">taskmanager.numberOfTaskSlots:</span> <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 程序默认并行计算的个数</span></span><br><span class="line"><span class="attr">parallelism.default:</span> <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 文件系统来源, 使用本地文件系统：&quot;file:///&quot;, 使用 HDFS 分布式文件系统：&quot;hdfs://mynamenode:12345&quot;</span></span><br><span class="line"><span class="comment"># fs.default-scheme</span></span><br><span class="line"><span class="comment"># fs.default-scheme: hdfs://localhost:9000</span></span><br></pre></td></tr></table></figure>

<h2 id="5-2-高可用"><a href="#5-2-高可用" class="headerlink" title="5.2 高可用"></a>5.2 高可用</h2><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 可以选择 &#x27;NONE&#x27; 或者 &#x27;zookeeper&#x27;.</span></span><br><span class="line"><span class="comment"># high-availability: zookeeper</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 文件系统路径，让 Flink 在高可用性设置中持久保存元数据</span></span><br><span class="line"><span class="comment"># high-availability.storageDir: hdfs:///flink/ha/</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># zookeeper 集群中仲裁者的机器 ip 和 port 端口号, host1:clientPort,host2:clientPort,...</span></span><br><span class="line"><span class="comment"># high-availability.zookeeper.quorum: localhost:2181</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 默认是 open，如果 zookeeper security 启用了该值会更改成 creator</span></span><br><span class="line"><span class="comment"># &quot;creator&quot; (ZOO_CREATE_ALL_ACL) 或者 &quot;open&quot; (ZOO_OPEN_ACL_UNSAFE)</span></span><br><span class="line"><span class="comment"># high-availability.zookeeper.client.acl: open</span></span><br></pre></td></tr></table></figure>

<h2 id="5-3-容错和检查点配置"><a href="#5-3-容错和检查点配置" class="headerlink" title="5.3 容错和检查点配置"></a>5.3 容错和检查点配置</h2><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 用于存储和检查点状态，可选值：&#x27;jobmanager&#x27;, &#x27;filesystem&#x27;, &#x27;rocksdb&#x27;, or the &lt;class-name-of-factory&gt;</span></span><br><span class="line"><span class="comment"># state.backend: filesystem</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 存储检查点的数据文件和元数据的默认目录</span></span><br><span class="line"><span class="comment"># state.checkpoints.dir: hdfs://namenode-host:port/flink-checkpoints</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># savepoints 的默认目标目录(可选)</span></span><br><span class="line"><span class="comment"># state.savepoints.dir: hdfs://namenode-host:port/flink-checkpoints</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 用于启用/禁用增量 checkpoints 的标志</span></span><br><span class="line"><span class="comment"># state.backend.incremental: false</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 故障转移策略，即作业计算如何从任务故障中恢复。</span></span><br><span class="line"><span class="comment"># 仅重新启动可能已受任务故障影响的任务，通常包括下游任务和潜在的上游任务（如果它们产生的数据不再可供使用）。</span></span><br><span class="line"><span class="attr">jobmanager.execution.failover-strategy:</span> <span class="string">region</span></span><br></pre></td></tr></table></figure>

<h2 id="5-4-Restful-和-web-前端配置"><a href="#5-4-Restful-和-web-前端配置" class="headerlink" title="5.4 Restful 和 web 前端配置"></a>5.4 Restful 和 web 前端配置</h2><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 基于 Web 的运行时监视器侦听的地址.</span></span><br><span class="line"><span class="comment">#jobmanager.web.address: 0.0.0.0</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># rest 端口，如果没有指定 rest.bind-port 的话会使用这个配置</span></span><br><span class="line"><span class="attr">rest.port:</span> <span class="number">8081</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># rest 客户端地址</span></span><br><span class="line"><span class="attr">rest.address:</span> <span class="number">0.0</span><span class="number">.0</span><span class="number">.0</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># rest 和 web 服务的端口范围</span></span><br><span class="line"><span class="attr">rest.bind-port:</span> <span class="number">8080</span><span class="number">-8090</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># rest 和 web 服务的地址</span></span><br><span class="line"><span class="attr">rest.bind-address:</span> <span class="number">0.0</span><span class="number">.0</span><span class="number">.0</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 标记以指定是否从基于Web的运行时监视器启用作业提交。 取消注释以禁用。</span></span><br><span class="line"><span class="comment"># jobmanager.web.submit.enable: false</span></span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h2 id="5-5-高级配置"><a href="#5-5-高级配置" class="headerlink" title="5.5 高级配置"></a>5.5 高级配置</h2><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># io.tmp.dirs: /tmp</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 是否应在 TaskManager 启动时预先分配 TaskManager 管理的内存</span></span><br><span class="line"><span class="comment"># taskmanager.memory.preallocate: false</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 类加载解析顺序，是先检查用户代码 jar（“child-first”）还是应用程序类路径（“parent-first”）。 默认设置指示首先从用户代码 jar 加载类</span></span><br><span class="line"><span class="comment"># classloader.resolve-order: child-first</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 用于网络缓冲区的 JVM 内存的分数。 </span></span><br><span class="line"><span class="comment"># 这决定了 TaskManager 可以同时拥有多少流数据交换通道以及通道缓冲的程度。 </span></span><br><span class="line"><span class="comment"># 如果作业被拒绝或者您收到系统没有足够缓冲区的警告，请增加此值或下面的最小/最大值。 </span></span><br><span class="line"></span><br><span class="line"><span class="comment"># taskmanager.memory.network.fraction: 0.1</span></span><br><span class="line"><span class="comment"># taskmanager.memory.network.min: 64mb</span></span><br><span class="line"><span class="comment"># taskmanager.memory.network.max: 1gb</span></span><br></pre></td></tr></table></figure>

<h2 id="5-6-Flink-集群安全配置"><a href="#5-6-Flink-集群安全配置" class="headerlink" title="5.6 Flink 集群安全配置"></a>5.6 Flink 集群安全配置</h2><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 指示是否从 Kerberos ticket 缓存中读取</span></span><br><span class="line"><span class="comment"># security.kerberos.login.use-ticket-cache: true</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 包含用户凭据的 Kerberos 密钥表文件的绝对路径</span></span><br><span class="line"><span class="comment"># security.kerberos.login.keytab: /path/to/kerberos/keytab</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 与 keytab 关联的 Kerberos 主体名称</span></span><br><span class="line"><span class="comment"># security.kerberos.login.principal: flink-user</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 以逗号分隔的登录上下文列表，用于提供 Kerberos 凭据（例如，`Client，KafkaClient`使用凭证进行 ZooKeeper 身份验证和 Kafka 身份验证）</span></span><br><span class="line"><span class="comment"># security.kerberos.login.contexts: Client,KafkaClient</span></span><br></pre></td></tr></table></figure>

<h2 id="5-7-Zookeeper-安全配置"><a href="#5-7-Zookeeper-安全配置" class="headerlink" title="5.7 Zookeeper 安全配置"></a>5.7 Zookeeper 安全配置</h2><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 覆盖以下配置以提供自定义 ZK 服务名称</span></span><br><span class="line"><span class="comment"># zookeeper.sasl.service-name: zookeeper</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 该配置必须匹配 &quot;security.kerberos.login.contexts&quot; 中的列表（含有一个）</span></span><br><span class="line"><span class="comment"># zookeeper.sasl.login-context-name: Client</span></span><br></pre></td></tr></table></figure>

<h2 id="5-8-HistoryServer"><a href="#5-8-HistoryServer" class="headerlink" title="5.8 HistoryServer"></a>5.8 HistoryServer</h2><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 你可以通过 bin/historyserver.sh (start|stop) 命令启动和关闭 HistoryServer</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将已完成的作业上传到的目录</span></span><br><span class="line"><span class="comment"># jobmanager.archive.fs.dir: hdfs:///completed-jobs/</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 基于 Web 的 HistoryServer 的地址</span></span><br><span class="line"><span class="comment"># historyserver.web.address: 0.0.0.0</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 基于 Web 的 HistoryServer 的端口号</span></span><br><span class="line"><span class="comment"># historyserver.web.port: 8082</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 以逗号分隔的目录列表，用于监视已完成的作业</span></span><br><span class="line"><span class="comment"># historyserver.archive.fs.dir: hdfs:///completed-jobs/</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 刷新受监控目录的时间间隔（以毫秒为单位）</span></span><br><span class="line"><span class="comment"># historyserver.archive.fs.refresh-interval: 10000</span></span><br></pre></td></tr></table></figure>

<h1 id="6-masters"><a href="#6-masters" class="headerlink" title="6 masters"></a>6 masters</h1><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">localhost:8081</span><br></pre></td></tr></table></figure>

<h1 id="7-workers"><a href="#7-workers" class="headerlink" title="7 workers"></a>7 workers</h1><p>里面是每个 worker 节点的 IP/Hostname，每一个 worker 结点之后都会运行一个 TaskManager，一个一行。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">localhost</span><br></pre></td></tr></table></figure>



<h1 id="8-zoo-cfg-可选"><a href="#8-zoo-cfg-可选" class="headerlink" title="8 zoo.cfg(可选)"></a>8 zoo.cfg(可选)</h1><p>Flink 自带的 zookeeper，如果使用外部独立的zookeeper</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"># 每个 tick 的毫秒数</span><br><span class="line">tickTime&#x3D;2000</span><br><span class="line"></span><br><span class="line"># 初始同步阶段可以采用的 tick 数</span><br><span class="line">initLimit&#x3D;10</span><br><span class="line"></span><br><span class="line"># 在发送请求和获取确认之间可以传递的 tick 数</span><br><span class="line">syncLimit&#x3D;5</span><br><span class="line"></span><br><span class="line"># 存储快照的目录</span><br><span class="line"># dataDir&#x3D;&#x2F;tmp&#x2F;zookeeper</span><br><span class="line"></span><br><span class="line"># 客户端将连接的端口</span><br><span class="line">clientPort&#x3D;2181</span><br><span class="line"></span><br><span class="line"># ZooKeeper quorum peers</span><br><span class="line">server.1&#x3D;localhost:2888:3888</span><br><span class="line"># server.2&#x3D;host:peer-port:leader-port</span><br></pre></td></tr></table></figure>



<h1 id="参考资源"><a href="#参考资源" class="headerlink" title="参考资源"></a>参考资源</h1><ul>
<li><a target="_blank" rel="noopener" href="https://blog.csdn.net/oTengYue/article/details/102689538">一文弄懂Flink基础理论</a></li>
<li><a target="_blank" rel="noopener" href="https://bigjar.github.io/2018/11/13/Akka%E5%9C%A8Flink%E4%B8%AD%E7%9A%84%E5%BA%94%E7%94%A8/#Akka%E5%92%8CActor%E6%A8%A1%E5%9E%8B">Akka在Flink中的应用</a></li>
<li><a target="_blank" rel="noopener" href="https://cwiki.apache.org/confluence/display/FLINK/Akka+and+Actors">Akka and Actors</a></li>
<li><a target="_blank" rel="noopener" href="https://blog.csdn.net/qq_19427611/article/details/80378273">【Flink系列】之 Akka和Actors</a></li>
</ul>

    </div>

    
    
    

    <footer class="post-footer">
          <div class="post-tags">
              <a href="/tags/flink/" rel="tag"># flink</a>
          </div>

        

          <div class="post-nav">
            <div class="post-nav-item">
                <a href="/2021/04/07/RestTemplateAndOkHttp3/" rel="prev" title="RestTemplateAndOkHttp3">
                  <i class="fa fa-chevron-left"></i> RestTemplateAndOkHttp3
                </a>
            </div>
            <div class="post-nav-item">
                <a href="/2021/04/08/flink%E8%B5%84%E6%96%99%E6%B1%87%E6%80%BB/" rel="next" title="flink资料汇总">
                  flink资料汇总 <i class="fa fa-chevron-right"></i>
                </a>
            </div>
          </div>
    </footer>
  </article>
</div>







<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      const activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      const commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>
</div>
  </main>

  <footer class="footer">
    <div class="footer-inner">


<div class="copyright">
  &copy; 
  <span itemprop="copyrightYear">2022</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">一年春又来</span>
</div>
  <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://theme-next.js.org/mist/" class="theme-link" rel="noopener" target="_blank">NexT.Mist</a> 强力驱动
  </div>

    </div>
  </footer>

  
  <script src="https://cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
<script src="/js/utils.js"></script><script src="/js/motion.js"></script><script src="/js/schemes/muse.js"></script><script src="/js/next-boot.js"></script>

  
<script src="/js/local-search.js"></script>






  





</body>
</html>
